import sys
from multiprocessing import Pool
from 转转网.channel_extract import channel_list
from 转转网.page_spider import get_links
from 转转网.page_spider import get_info
from 转转网.page_spider import tongcheng_info
from 转转网.page_spider import tongcheng_url
db_urls = [item['url'] for item in tongcheng_url.find()]
db_infos = [item['url'] for item in tongcheng_info.find()]

x = set(db_urls)
y = set(db_infos)

reset_urls = x - y

def get_all_links_from(channel):
    for num in range(1,101):
        get_links(channel,num)

if __name__ == '__main__':
    pool = Pool(processes=4)
    pool.map(get_all_links_from,channel_list.split())
    # pool.map(get_info,reset_urls)